package com.oliver.oliveraiagent.rag;

import org.springframework.ai.document.Document;
import org.springframework.ai.transformer.splitter.TokenTextSplitter;
import org.springframework.stereotype.Component;

import java.util.List;

/**
 * @ClassName MyTokenTextSpiltter
 * @Description TODO
 * @Author snow
 * @Date 2025/5/21 23:28
 **/

/**
 * 自定义的 Token 分词切割器
 *
 */
@Component
public class MyTokenTextSplitter {
    public List<Document> splitDocuments(List<Document> documents) {
        TokenTextSplitter splitter = new TokenTextSplitter();
        return splitter.apply(documents);
    }

    /**
     * 自定义切割文档
     * @param documents
     * @return
     */
    public List<Document> splitCustomized(List<Document> documents) {
        TokenTextSplitter splitter = new TokenTextSplitter(200,
                100,
                10,
                5000,
                true);
        return splitter.apply(documents);
    }
}

